diff --git a/cypress/fixtures/save-task-info.json b/cypress/fixtures/save-task-info.json
new file mode 100644
--- /dev/null
+++ b/cypress/fixtures/save-task-info.json
@@ -0,0 +1,16 @@
+{
+ "scheduled": "2020-06-24T11:48:12.561643+00:00",
+ "started": "2020-06-24T12:59:11.103188+00:00",
+ "ended": "2020-06-24T12:59:12.065313+00:00",
+ "status": "eventful",
+ "type": "load-git",
+ "arguments": {
+ "args": [],
+ "kwargs": {
+ "url": "https://gitlab.inria.fr/solverstack/maphys/maphys/"
+ }
+ },
+ "duration": "1.0600971020758152",
+ "message": "[2020-06-24 12:59:12,063: INFO/ForkPoolWorker-161] Task swh.loader.git.tasks.UpdateGitRepository[4ff8b555-9535-4e75-b8ec-8e76165e14ec] succeeded in 1.0600971020758152s: {'status': 'eventful'}",
+ "name": "swh.loader.git.tasks.UpdateGitRepository"
+}
\ No newline at end of file
diff --git a/cypress/integration/origin-save.spec.js b/cypress/integration/origin-save.spec.js
--- a/cypress/integration/origin-save.spec.js
+++ b/cypress/integration/origin-save.spec.js
@@ -159,34 +159,76 @@
});
it('should display origin save info in the requests table', function() {
- cy.fixture('origin-save').then(originSaveJSON => {
- cy.route('GET', '/save/requests/list/**', originSaveJSON);
- cy.get('#swh-origin-save-requests-list-tab').click();
- cy.get('tbody tr').then(rows => {
- let i = 0;
- for (let row of rows) {
- const cells = row.cells;
- const requestDateStr = new Date(originSaveJSON.data[i].save_request_date).toLocaleString();
- const saveStatus = originSaveJSON.data[i].save_task_status;
- assert.equal($(cells[0]).text(), requestDateStr);
- assert.equal($(cells[1]).text(), originSaveJSON.data[i].visit_type);
- let html = '';
- if (saveStatus === 'succeed') {
- let browseOriginUrl = `${this.Urls.browse_origin()}?origin_url=${originSaveJSON.data[i].origin_url}`;
- browseOriginUrl += `&timestamp=${originSaveJSON.data[i].visit_date}`;
- html += `<a href="${browseOriginUrl}">${originSaveJSON.data[i].origin_url}</a>`;
- } else {
- html += originSaveJSON.data[i].origin_url;
- }
- html += ` <a href="${originSaveJSON.data[i].origin_url}">`;
- html += '<i class="mdi mdi-open-in-new" aria-hidden="true"></i></a>';
- assert.equal($(cells[2]).html(), html);
- assert.equal($(cells[3]).text(), originSaveJSON.data[i].save_request_status);
- assert.equal($(cells[4]).text(), saveStatus);
- ++i;
+ cy.fixture('origin-save').as('originSaveJSON');
+ cy.route('GET', '/save/requests/list/**', '@originSaveJSON');
+ cy.get('#swh-origin-save-requests-list-tab').click();
+ cy.get('tbody tr').then(rows => {
+ let i = 0;
+ for (let row of rows) {
+ const cells = row.cells;
+ const requestDateStr = new Date(this.originSaveJSON.data[i].save_request_date).toLocaleString();
+ const saveStatus = this.originSaveJSON.data[i].save_task_status;
+ assert.equal($(cells[0]).text(), requestDateStr);
+ assert.equal($(cells[1]).text(), this.originSaveJSON.data[i].visit_type);
+ let html = '';
+ if (saveStatus === 'succeed') {
+ let browseOriginUrl = `${this.Urls.browse_origin()}?origin_url=${this.originSaveJSON.data[i].origin_url}`;
+ browseOriginUrl += `&timestamp=${this.originSaveJSON.data[i].visit_date}`;
+ html += `<a href="${browseOriginUrl}">${this.originSaveJSON.data[i].origin_url}</a>`;
+ } else {
+ html += this.originSaveJSON.data[i].origin_url;
}
- });
+ html += ` <a href="${this.originSaveJSON.data[i].origin_url}">`;
+ html += '<i class="mdi mdi-open-in-new" aria-hidden="true"></i></a>';
+ assert.equal($(cells[2]).html(), html);
+ assert.equal($(cells[3]).text(), this.originSaveJSON.data[i].save_request_status);
+ assert.equal($(cells[4]).text(), saveStatus);
+ ++i;
+ }
});
});
+ it('should display/close task info popover when clicking on the info button', function() {
+ cy.fixture('origin-save').as('originSaveJSON');
+ cy.fixture('save-task-info').as('saveTaskInfoJSON');
+ cy.route('GET', '/save/requests/list/**', '@originSaveJSON');
+ cy.route('GET', '/save/task/info/**', '@saveTaskInfoJSON');
+
+ cy.get('#swh-origin-save-requests-list-tab').click();
+ cy.get('.swh-save-request-info')
+ .eq(0)
+ .click();
+
+ cy.get('.swh-save-request-info-popover')
+ .should('be.visible');
+
+ cy.get('.swh-save-request-info')
+ .eq(0)
+ .click();
+
+ cy.get('.swh-save-request-info-popover')
+ .should('not.be.visible');
+ });
+
+ it('should hide task info popover when clicking on the close button', function() {
+ cy.fixture('origin-save').as('originSaveJSON');
+ cy.fixture('save-task-info').as('saveTaskInfoJSON');
+ cy.route('GET', '/save/requests/list/**', '@originSaveJSON');
+ cy.route('GET', '/save/task/info/**', '@saveTaskInfoJSON');
+
+ cy.get('#swh-origin-save-requests-list-tab').click();
+ cy.get('.swh-save-request-info')
+ .eq(0)
+ .click();
+
+ cy.get('.swh-save-request-info-popover')
+ .should('be.visible');
+
+ cy.get('.swh-save-request-info-close')
+ .click();
+
+ cy.get('.swh-save-request-info-popover')
+ .should('not.be.visible');
+ });
+
});
diff --git a/swh/web/admin/origin_save.py b/swh/web/admin/origin_save.py
--- a/swh/web/admin/origin_save.py
+++ b/swh/web/admin/origin_save.py
@@ -23,7 +23,6 @@
from swh.web.common.origin_save import (
create_save_origin_request,
- get_save_origin_task_info,
SAVE_REQUEST_PENDING,
SAVE_REQUEST_REJECTED,
)
@@ -214,16 +213,3 @@
entry.delete()
status_code = 200
return HttpResponse(status=status_code)
-
-
-@admin_route(
- r"origin/save/task/info/(?P<save_request_id>.+)/",
- view_name="admin-origin-save-task-info",
-)
-@staff_member_required(view_func=None, login_url=settings.LOGIN_URL)
-def _save_origin_task_info(request, save_request_id):
- request_info = get_save_origin_task_info(save_request_id)
- for date_field in ("scheduled", "started", "ended"):
- if date_field in request_info and request_info[date_field] is not None:
- request_info[date_field] = request_info[date_field].isoformat()
- return HttpResponse(json.dumps(request_info), content_type="application/json")
diff --git a/swh/web/assets/src/bundles/admin/origin-save.js b/swh/web/assets/src/bundles/admin/origin-save.js
--- a/swh/web/assets/src/bundles/admin/origin-save.js
+++ b/swh/web/assets/src/bundles/admin/origin-save.js
@@ -150,7 +150,7 @@
render: (data, type, row) => {
if (row.save_task_status === 'succeed' || row.save_task_status === 'failed') {
return '<i class="mdi mdi-information-outline swh-save-request-info" aria-hidden="true" style="cursor: pointer"' +
- `onclick="swh.admin.displaySaveRequestInfo(event, ${row.id})"></i>`;
+ `onclick="swh.save.displaySaveRequestInfo(event, ${row.id})"></i>`;
} else {
return '';
}
@@ -359,93 +359,3 @@
export function removeRejectedOriginSaveRequest() {
removeOriginSaveRequest(rejectedSaveRequestsTable);
}
-
-export function displaySaveRequestInfo(event, saveRequestId) {
- event.stopPropagation();
- const saveRequestTaskInfoUrl = Urls.admin_origin_save_task_info(saveRequestId);
- $('.swh-save-request-info').popover('dispose');
- $(event.target).popover({
- 'title': 'Save request task information',
- 'content': `<div class="swh-popover">
- <div class="text-center">
- <img src=${swhSpinnerSrc}></img>
- <p>Fetching task information ...</p>
- </div>
- </div>`,
- 'html': true,
- 'placement': 'left',
- 'sanitizeFn': swh.webapp.filterXSS
- });
- $(event.target).popover('show');
- fetch(saveRequestTaskInfoUrl)
- .then(response => response.json())
- .then(saveRequestTaskInfo => {
- let content;
- if ($.isEmptyObject(saveRequestTaskInfo)) {
- content = 'Not available';
- } else {
- let saveRequestInfo = [];
- saveRequestInfo.push({
- key: 'Task type',
- value: saveRequestTaskInfo.type
- });
- if (saveRequestTaskInfo.hasOwnProperty('task_name')) {
- saveRequestInfo.push({
- key: 'Task name',
- value: saveRequestTaskInfo.name
- });
- }
- saveRequestInfo.push({
- key: 'Task arguments',
- value: JSON.stringify(saveRequestTaskInfo.arguments, null, 2)
- });
- saveRequestInfo.push({
- key: 'Task id',
- value: saveRequestTaskInfo.id
- });
- saveRequestInfo.push({
- key: 'Task backend id',
- value: saveRequestTaskInfo.backend_id
- });
- saveRequestInfo.push({
- key: 'Task scheduling date',
- value: new Date(saveRequestTaskInfo.scheduled).toLocaleString()
- });
- saveRequestInfo.push({
- key: 'Task termination date',
- value: new Date(saveRequestTaskInfo.ended).toLocaleString()
- });
- if (saveRequestTaskInfo.hasOwnProperty('duration')) {
- saveRequestInfo.push({
- key: 'Task duration',
- value: saveRequestTaskInfo.duration + ' s'
- });
- }
- if (saveRequestTaskInfo.hasOwnProperty('worker')) {
- saveRequestInfo.push({
- key: 'Task executor',
- value: saveRequestTaskInfo.worker
- });
- }
- if (saveRequestTaskInfo.hasOwnProperty('message')) {
- saveRequestInfo.push({
- key: 'Task log',
- value: saveRequestTaskInfo.message
- });
- }
- content = '<table class="table"><tbody>';
- for (let info of saveRequestInfo) {
- content +=
- `<tr>
- <th class="swh-metadata-table-row swh-metadata-table-key">${info.key}</th>
- <td class="swh-metadata-table-row swh-metadata-table-value">
- <pre>${info.value}</pre>
- </td>
- </tr>`;
- }
- content += '</tbody></table>';
- }
- $('.swh-popover').html(content);
- $(event.target).popover('update');
- });
-}
diff --git a/swh/web/assets/src/bundles/save/index.js b/swh/web/assets/src/bundles/save/index.js
--- a/swh/web/assets/src/bundles/save/index.js
+++ b/swh/web/assets/src/bundles/save/index.js
@@ -108,6 +108,18 @@
{
data: 'save_task_status',
name: 'loading_task_status'
+ },
+ {
+ name: 'info',
+ render: (data, type, row) => {
+ if (row.save_task_status === 'succeed' || row.save_task_status === 'failed') {
+ return `<i class="mdi mdi-information-outline swh-save-request-info" ` +
+ 'aria-hidden="true" style="cursor: pointer"' +
+ `onclick="swh.save.displaySaveRequestInfo(event, ${row.id})"></i>`;
+ } else {
+ return '';
+ }
+ }
}
],
scrollY: '50vh',
@@ -127,8 +139,9 @@
window.location.hash = '#requests';
});
- $('#swh-origin-save-request-create-tab').on('shown.bs.tab', () => {
+ $('#swh-origin-save-request-help-tab').on('shown.bs.tab', () => {
removeUrlFragment();
+ $('.swh-save-request-info').popover('dispose');
});
let saveRequestAcceptedAlert = htmlAlert(
@@ -298,3 +311,123 @@
});
});
}
+
+export function displaySaveRequestInfo(event, saveRequestId) {
+ event.stopPropagation();
+ const saveRequestTaskInfoUrl = Urls.origin_save_task_info(saveRequestId);
+ // close popover when clicking again on the info icon
+ if ($(event.target).data('bs.popover')) {
+ $(event.target).popover('dispose');
+ return;
+ }
+ $('.swh-save-request-info').popover('dispose');
+ $(event.target).popover({
+ animation: false,
+ boundary: 'viewport',
+ container: 'body',
+ title: 'Save request task information ' +
+ '<i style="cursor: pointer; position: absolute; right: 1rem;" ' +
+ `class="mdi mdi-close swh-save-request-info-close"></i>`,
+ content: `<div class="swh-popover swh-save-request-info-popover">
+ <div class="text-center">
+ <img src=${swhSpinnerSrc}></img>
+ <p>Fetching task information ...</p>
+ </div>
+ </div>`,
+ html: true,
+ placement: 'left',
+ sanitizeFn: swh.webapp.filterXSS
+ });
+
+ $(event.target).on('shown.bs.popover', function() {
+ const popoverId = $(this).attr('aria-describedby');
+ $(`#${popoverId} .mdi-close`).click(() => {
+ $(this).popover('dispose');
+ });
+ });
+
+ $(event.target).popover('show');
+ fetch(saveRequestTaskInfoUrl)
+ .then(response => response.json())
+ .then(saveRequestTaskInfo => {
+ let content;
+ if ($.isEmptyObject(saveRequestTaskInfo)) {
+ content = 'Not available';
+ } else {
+ let saveRequestInfo = [];
+ if (saveRequestTaskInfo.type) {
+ saveRequestInfo.push({
+ key: 'Task type',
+ value: saveRequestTaskInfo.type
+ });
+ }
+ if (saveRequestTaskInfo.arguments) {
+ saveRequestInfo.push({
+ key: 'Task arguments',
+ value: JSON.stringify(saveRequestTaskInfo.arguments, null, 2)
+ });
+ }
+ if (saveRequestTaskInfo.id) {
+ saveRequestInfo.push({
+ key: 'Task id',
+ value: saveRequestTaskInfo.id
+ });
+ }
+ if (saveRequestTaskInfo.backend_id) {
+ saveRequestInfo.push({
+ key: 'Task backend id',
+ value: saveRequestTaskInfo.backend_id
+ });
+ }
+ if (saveRequestTaskInfo.scheduled) {
+ saveRequestInfo.push({
+ key: 'Task scheduling date',
+ value: new Date(saveRequestTaskInfo.scheduled).toLocaleString()
+ });
+ }
+ if (saveRequestTaskInfo.started) {
+ saveRequestInfo.push({
+ key: 'Task start date',
+ value: new Date(saveRequestTaskInfo.started).toLocaleString()
+ });
+ }
+ if (saveRequestTaskInfo.ended) {
+ saveRequestInfo.push({
+ key: 'Task termination date',
+ value: new Date(saveRequestTaskInfo.ended).toLocaleString()
+ });
+ }
+ if (saveRequestTaskInfo.duration) {
+ saveRequestInfo.push({
+ key: 'Task duration',
+ value: saveRequestTaskInfo.duration + ' seconds'
+ });
+ }
+ if (saveRequestTaskInfo.worker) {
+ saveRequestInfo.push({
+ key: 'Task executor',
+ value: saveRequestTaskInfo.worker
+ });
+ }
+ if (saveRequestTaskInfo.message) {
+ saveRequestInfo.push({
+ key: 'Task log',
+ value: saveRequestTaskInfo.message
+ });
+ }
+ content = '<table class="table"><tbody>';
+ for (let info of saveRequestInfo) {
+ content +=
+ `<tr>
+ <th class="swh-metadata-table-row swh-metadata-table-key">${info.key}</th>
+ <td class="swh-metadata-table-row swh-metadata-table-value">
+ <pre>${info.value}</pre>
+ </td>
+ </tr>`;
+ }
+ content += '</tbody></table>';
+ }
+ $('.swh-popover').html(content);
+ $(event.target).popover('update');
+ });
+}
diff --git a/swh/web/common/origin_save.py b/swh/web/common/origin_save.py
--- a/swh/web/common/origin_save.py
+++ b/swh/web/common/origin_save.py
@@ -8,6 +8,7 @@
from itertools import product
import json
import logging
+from typing import Any, Dict
from django.core.exceptions import ObjectDoesNotExist
from django.core.exceptions import ValidationError
@@ -424,7 +425,9 @@
return get_save_origin_requests_from_queryset(sors)
-def get_save_origin_task_info(save_request_id):
+def get_save_origin_task_info(
+ save_request_id: int, full_info: bool = True
+) -> Dict[str, Any]:
"""
Get detailed information about an accepted save origin request
and its associated loading task.
@@ -433,10 +436,11 @@
from the scheduler database, returns an empty dictionary.
Args:
- save_request_id (int): identifier of a save origin request
+ save_request_id: identifier of a save origin request
+ full_info: whether to return detailed info for staff users
Returns:
- dict: A dictionary with the following keys:
+ A dictionary with the following keys:
- **type**: loading task type
- **arguments**: loading task arguments
@@ -474,7 +478,6 @@
task_run["id"] = task_run["task"]
del task_run["task"]
del task_run["metadata"]
- del task_run["started"]
es_workers_index_url = config.get_config()["es_workers_index_url"]
if not es_workers_index_url:
@@ -487,8 +490,8 @@
else:
min_ts = save_request.request_date
max_ts = min_ts + timedelta(days=30)
- min_ts = int(min_ts.timestamp()) * 1000
- max_ts = int(max_ts.timestamp()) * 1000
+ min_ts_unix = int(min_ts.timestamp()) * 1000
+ max_ts_unix = int(max_ts.timestamp()) * 1000
save_task_status = _save_task_status[task["status"]]
priority = "3" if save_task_status == SAVE_TASK_FAILED else "6"
@@ -501,8 +504,8 @@
{
"range": {
"@timestamp": {
- "gte": min_ts,
- "lte": max_ts,
+ "gte": min_ts_unix,
+ "lte": max_ts_unix,
"format": "epoch_millis",
}
}
@@ -537,6 +540,21 @@
logger.warning("Request to Elasticsearch failed\n%s", exc)
sentry_sdk.capture_exception(exc)
+ if not full_info:
+ for field in ("id", "backend_id", "worker"):
+ # remove some staff only fields
+ task_run.pop(field, None)
+ if "message" in task_run and "Loading failure" in task_run["message"]:
+ # hide traceback for non staff users, only display exception
+ message_lines = task_run["message"].split("\n")
+ message = ""
+ for line in message_lines:
+ if line.startswith("Traceback"):
+ break
+ message += f"{line}\n"
+ message += message_lines[-1]
+ task_run["message"] = message
+
return task_run
diff --git a/swh/web/misc/origin_save.py b/swh/web/misc/origin_save.py
--- a/swh/web/misc/origin_save.py
+++ b/swh/web/misc/origin_save.py
@@ -19,6 +19,7 @@
create_save_origin_request,
get_savable_visit_types,
get_save_origin_requests_from_queryset,
+ get_save_origin_task_info,
)
from swh.web.common.utils import EnforceCSRFAuthentication
@@ -101,6 +102,16 @@
return HttpResponse(table_data_json, content_type="application/json")
+def _save_origin_task_info(request, save_request_id):
+ request_info = get_save_origin_task_info(
+ save_request_id, full_info=request.user.is_staff
+ )
+ for date_field in ("scheduled", "started", "ended"):
+ if date_field in request_info and request_info[date_field] is not None:
+ request_info[date_field] = request_info[date_field].isoformat()
+ return HttpResponse(json.dumps(request_info), content_type="application/json")
+
+
urlpatterns = [
url(r"^save/$", _origin_save_view, name="origin-save"),
url(
@@ -114,4 +125,9 @@
_origin_save_requests_list,
name="origin-save-requests-list",
),
+ url(
+ r"^save/task/info/(?P<save_request_id>.+)/",
+ _save_origin_task_info,
+ name="origin-save-task-info",
+ ),
]
diff --git a/swh/web/templates/misc/origin-save.html b/swh/web/templates/misc/origin-save.html
--- a/swh/web/templates/misc/origin-save.html
+++ b/swh/web/templates/misc/origin-save.html
@@ -59,7 +59,7 @@
</div>
<ul class="nav nav-tabs" style="padding-left: 5px;">
- <li class="nav-item"><a class="nav-link active" data-toggle="tab" id="swh-origin-save-request-create-tab" href="#swh-origin-save-requests-create">Help</a></li>
+ <li class="nav-item"><a class="nav-link active" data-toggle="tab" id="swh-origin-save-request-help-tab" href="#swh-origin-save-requests-create">Help</a></li>
<li class="nav-item"><a class="nav-link" data-toggle="tab" id="swh-origin-save-requests-list-tab" href="#swh-origin-save-requests-list">Browse save requests</a></li>
</ul>
@@ -109,6 +109,7 @@
<th data-priority="1">Url</th>
<th data-priority="5">Request</th>
<th data-priority="2">Status</th>
+ <th data-priority="6">Info</th>
</tr>
</thead>
</table>
diff --git a/swh/web/tests/common/test_origin_save.py b/swh/web/tests/common/test_origin_save.py
--- a/swh/web/tests/common/test_origin_save.py
+++ b/swh/web/tests/common/test_origin_save.py
@@ -46,10 +46,15 @@
@pytest.mark.django_db
-def test_get_save_origin_task_info_with_es(mocker):
+def test_get_save_origin_task_full_info_with_es(mocker):
_get_save_origin_task_info_test(mocker, es_available=True)
+@pytest.mark.django_db
+def test_get_save_origin_task_info_with_es(mocker):
+ _get_save_origin_task_info_test(mocker, es_available=True, full_info=False)
+
+
@pytest.mark.django_db
def test_get_save_origin_task_info_without_es(mocker):
_get_save_origin_task_info_test(mocker, es_available=False)
@@ -72,7 +77,7 @@
if not task_archived
else None
)
- mock_scheduler.get_tasks.return_value = [task]
+ mock_scheduler.get_tasks.return_value = [dict(task) if task else None]
task_run = {
"backend_id": "f00c712c-e820-41ce-a07c-9bf8df914205",
@@ -84,11 +89,13 @@
"status": task_status,
"task": _task_id,
}
- mock_scheduler.get_task_runs.return_value = [task_run]
+ mock_scheduler.get_task_runs.return_value = [dict(task_run)]
return task, task_run
-def _get_save_origin_task_info_test(mocker, task_archived=False, es_available=True):
+def _get_save_origin_task_info_test(
+ mocker, task_archived=False, es_available=True, full_info=True
+):
swh_web_config = get_config()
if es_available:
@@ -111,7 +118,7 @@
task_exec_data = es_response["hits"]["hits"][-1]["_source"]
- sor_task_info = get_save_origin_task_info(sor.id)
+ sor_task_info = get_save_origin_task_info(sor.id, full_info=full_info)
expected_result = (
{
@@ -120,6 +127,7 @@
"id": task["id"],
"backend_id": task_run["backend_id"],
"scheduled": task_run["scheduled"],
+ "started": task_run["started"],
"ended": task_run["ended"],
"status": task_run["status"],
}
@@ -136,6 +144,20 @@
}
)
+ if not full_info:
+ expected_result.pop("id", None)
+ expected_result.pop("backend_id", None)
+ expected_result.pop("worker", None)
+ if "message" in expected_result:
+ message = ""
+ message_lines = expected_result["message"].split("\n")
+ for line in message_lines:
+ if line.startswith("Traceback"):
+ break
+ message += f"{line}\n"
+ message += message_lines[-1]
+ expected_result["message"] = message
+
assert sor_task_info == expected_result